import scrapy
from fake_headers import Headers
from ..items import DoubanTop250Item


class DoubanSpider(scrapy.Spider):
    name = 'DoubanSpider'

    # headers = Headers(
    #     browser='chrome',
    #     os='win',
    #     headers=True
    # )
    headers = {
        "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36"
    }

    # 手动发起请求
    def start_requests(self):
        url_format = 'https://movie.douban.com/top250?start={}&filter='
        for i in range(10):
            url = url_format.format(i*25)
            yield scrapy.Request(url=url, headers=self.headers)

    # 解析函数
    def parse(self, response):
        # print(response.text)
        list_li = response.xpath("//ol[@class='grid_view']/li")
        # print(list_li)
        for li in list_li:
            div_info = li.xpath(".//div[@class='info']")[0]

            title = div_info.xpath("./div[1]/a/span[1]/text()")[0].extract()
            # print(title)
            p = div_info.xpath("./div[2]/p/text()")[1].extract()
            splits = p.strip().split('/')
            year = splits[0].strip()  # 年份
            country = splits[1].strip()  # 国家
            category = splits[2].strip()  # 类型
            # print(year, country, category)

            # 评分
            score = div_info.xpath("./div[2]/div/span[2]/text()")[0].extract()
            # 评价人数
            comment_cnt = div_info.xpath("./div[2]/div/span[4]/text()")[0].extract().split("人")[0]
            # print(score,comment_cnt)

            # 影评
            try:
                comment = div_info.xpath("./div[2]/p[2]/span[@class='inq']/text()")[0].extract()
                # print(comment)
            except IndexError as e:
                comment = ''

            doubanItem = DoubanTop250Item()
            doubanItem['title'] = title
            doubanItem['year'] = year
            doubanItem['country'] = country
            doubanItem['category'] = category
            doubanItem['score'] = score
            doubanItem['comment_cnt'] = comment_cnt
            doubanItem['comment'] = comment

            yield doubanItem
